package cn.haut.online_read.utils;

import com.alibaba.fastjson.JSON;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.pipeline.ConsolePipeline;
import us.codecraft.webmagic.pipeline.ResultItemsCollectorPipeline;
import us.codecraft.webmagic.processor.PageProcessor;

import java.util.Map;

/**
 * @author xing
 * @date 2021-05-05 19:48
 */
public class CrawlerUtils {

    public static String getJsonResult(PageProcessor processor, String url) {
        Map<String, Object> result = getResult(processor, url);
//        return JSON.toJSONString();
        return ResponseUtils.addSuccess(result);
    }

    public static Map<String, Object> getResult(PageProcessor processor, String url) {
        ResultItemsCollectorPipeline ricp = new ResultItemsCollectorPipeline();
        Spider.create(processor)
                .addUrl(url)
//                .addPipeline(new ConsolePipeline())
                .addPipeline(ricp)
                .run();
        return ricp.getCollected().get(0).getAll();
    }
}
